

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>QDMA Linux Driver Design Flow &mdash; QDMA Linux Driver 2018.3 documentation</title>
  

  
  

  

  
  
    

  

  
  
    <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
  

  

  
        <link rel="index" title="Index"
              href="genindex.html"/>
        <link rel="search" title="Search" href="search.html"/>
    <link rel="top" title="QDMA Linux Driver 2018.3 documentation" href="index.html"/>
        <link rel="up" title="Developers Guide" href="devguide.html"/>
        <link rel="next" title="QDMA Linux Driver UseCases" href="qdma_usecases.html"/>
        <link rel="prev" title="Libqdma APIs" href="libqdma_apis.html"/> 

  
  <script src="_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

          
            <a href="index.html" class="icon icon-home"> QDMA Linux Driver
          

          
          </a>

          
            
            
              <div class="version">
                2018.3
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
                <p class="caption"><span class="caption-text">Table of Contents</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="features.html">QDMA Features</a></li>
<li class="toctree-l1"><a class="reference internal" href="system-requirements.html">System Requirements</a></li>
<li class="toctree-l1"><a class="reference internal" href="build.html">Building and Installing Software Stack</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="devguide.html">Developers Guide</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="libqdma_apis.html">Libqdma APIs</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">QDMA Linux Driver Design Flow</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#driver-initialization">Driver Initialization</a></li>
<li class="toctree-l3"><a class="reference internal" href="#driver-de-initialization">Driver De-Initialization</a></li>
<li class="toctree-l3"><a class="reference internal" href="#qdma-driver-data-structures">QDMA Driver Data Structures</a></li>
<li class="toctree-l3"><a class="reference internal" href="#device-initialization">Device Initialization</a></li>
<li class="toctree-l3"><a class="reference internal" href="#device-de-initialization">Device De-Initialization</a></li>
<li class="toctree-l3"><a class="reference internal" href="#configure-the-queues-for-pf-vf">Configure the Queues for PF/VF</a></li>
<li class="toctree-l3"><a class="reference internal" href="#add-a-queue">Add a Queue</a></li>
<li class="toctree-l3"><a class="reference internal" href="#start-a-queue">Start a Queue</a></li>
<li class="toctree-l3"><a class="reference internal" href="#stop-a-queue">Stop a Queue</a></li>
<li class="toctree-l3"><a class="reference internal" href="#delete-a-queue">Delete a Queue</a></li>
<li class="toctree-l3"><a class="reference internal" href="#read-write-from-a-queue">Read/Write from a Queue</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="qdma_usecases.html">QDMA Linux Driver UseCases</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="userguide.html">User Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="user-app.html">User Applications</a></li>
<li class="toctree-l1"><a class="reference internal" href="performance.html">QDMA Performance</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
        <a href="index.html">QDMA Linux Driver</a>
      </nav>


      
      <div class="wy-nav-content">
        <div class="rst-content">
          

 



<div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
    <li><a href="index.html">Docs</a> &raquo;</li>
      
          <li><a href="devguide.html">Developers Guide</a> &raquo;</li>
      
    <li>QDMA Linux Driver Design Flow</li>
      <li class="wy-breadcrumbs-aside">
        
          
            <a href="_sources/qdma_design.rst.txt" rel="nofollow"> View page source</a>
          
        
      </li>
  </ul>
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="qdma-linux-driver-design-flow">
<h1>QDMA Linux Driver Design Flow<a class="headerlink" href="#qdma-linux-driver-design-flow" title="Permalink to this headline">¶</a></h1>
<div class="section" id="driver-initialization">
<h2>Driver Initialization<a class="headerlink" href="#driver-initialization" title="Permalink to this headline">¶</a></h2>
<p>QDMA Linux Driver is designed to configure and control the PCI based QDMA device connected to a x86 Host system.
It is a loadable kernel module which has three main components</p>
<ul>
<li><dl class="first docutils">
<dt>libqdma</dt>
<dd><p class="first"><em>libqdma</em> is a library which provides the APIs to manage the functions, queues and mailbox communication.
It creates multiple threads per each available core in the x86 system to parallely manage these entities.</p>
<ul class="simple">
<li><strong>main thread</strong>: responsible for accepting the requets from clients and submit the request to HW</li>
<li><strong>completion status processing thread</strong>: respondible for processing the MM and ST completions</li>
<li><strong>completion staus pending monitor thread</strong>: responsible for monitoring the completion status pending requests</li>
</ul>
<p class="last">Apart from creating the threads, libqdma also initializes the framework required for handling the legacy interrupts and
debugfs framework to collect the debug status information during runtime.</p>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt>drv</dt>
<dd><p class="first last"><em>drv</em> is a character device created as a wrapper around libqdma to enable the applications to perform read and write operations on the queues.
It also manages multiple devices attached to a single Host system.
It creates a list of physical devices attached to the Host system and manages the requests comming from
appilications to read/write to these devices.</p>
</dd>
</dl>
</li>
<li><dl class="first docutils">
<dt>xlnx_nl</dt>
<dd><p class="first last"><em>xlnx_nl</em> module creates and registers with netlink socket interface to provide various netlink interfcaes to perform device,
queue and function related operations</p>
</dd>
</dl>
</li>
</ul>
<p>During the module_init, it initializes these 3 main components and creates a PCI device interface i.e. <em>struct pci_driver</em>.</p>
<div class="highlight-none notranslate"><div class="highlight"><pre>static struct pci_driver pci_driver = {
        .name = DRV_MODULE_NAME,

        .id_table = pci_ids,

        .probe = probe_one,

        .remove = remove_one,

#if defined(CONFIG_PCI_IOV) &amp;&amp; !defined(__QDMA_VF__)
        .sriov_configure = sriov_config,
#endif
        .err_handler = &amp;qdma_err_handler,

};
</pre></div>
</div>
<p>Using the PCI device interface, driver lists the possible devices as par of the <code class="docutils literal notranslate"><span class="pre">pci_ids</span></code> and provides the probe and remove handler.
These handlers are called by each device in the list.
QDMA IP supports SRIOV and error handling functionality as well and QDMA driver enables the functions to configure SRIOV
using <code class="docutils literal notranslate"><span class="pre">sriov_config</span></code>  and <code class="docutils literal notranslate"><span class="pre">qdma_err_handler</span></code> handlers respectively.</p>
<img alt="_images/qdma_mod_init.PNG" class="align-center" src="_images/qdma_mod_init.PNG" />
</div>
<div class="section" id="driver-de-initialization">
<h2>Driver De-Initialization<a class="headerlink" href="#driver-de-initialization" title="Permalink to this headline">¶</a></h2>
<p>During the module_exit, all the resources created during initialization are released.</p>
<ul class="simple">
<li><em>libqdma</em> deletes all the created threads</li>
<li><em>drv</em> delete the physical device list</li>
<li><em>xlnx_nl</em> unregisters the xnl_family socket registered with libnl interface</li>
</ul>
<img alt="_images/qdma_mod_exit.PNG" class="align-center" src="_images/qdma_mod_exit.PNG" />
</div>
<div class="section" id="qdma-driver-data-structures">
<h2>QDMA Driver Data Structures<a class="headerlink" href="#qdma-driver-data-structures" title="Permalink to this headline">¶</a></h2>
<p>QDMA Linux Driver manages the all the qdma PCI devices with matching vendor id and device id listed in <code class="docutils literal notranslate"><span class="pre">pci_ids</span></code> table listed in <code class="docutils literal notranslate"><span class="pre">pci_ids.h</span></code>.
Each physical device connected over PCI to the x86 Host system is identified by the PCI bus number.
Within each physical device, It can have multiple functions and each function is treated as a PCI device and identified by the function number.</p>
<p><code class="docutils literal notranslate"><span class="pre">libqdma</span></code> treats each function as a dma device and represents them using <code class="docutils literal notranslate"><span class="pre">xlnx_dma_dev(xdev)</span></code>.
During initialization, qdma driver probes each PCI function and creates the corresponding <code class="docutils literal notranslate"><span class="pre">xdev</span></code>.
<code class="docutils literal notranslate"><span class="pre">xdev</span></code> is a driver level software structure maintained to preserve the software level details to manage a function.
<code class="docutils literal notranslate"><span class="pre">libqdma</span></code> also creates another data structure <code class="docutils literal notranslate"><span class="pre">qdma_dev(qdev)</span></code> corresponds to each function to keep track of the
h2c and c2h descriptor lists associated with each function. <code class="docutils literal notranslate"><span class="pre">qdma_descq(descq)</span></code> is a data structure maintained
for each queue in the system. <code class="docutils literal notranslate"><span class="pre">qdev</span></code> tracks the the list of queues associated with a function uisng <code class="docutils literal notranslate"><span class="pre">descq</span></code> and it inturn stores <code class="docutils literal notranslate"><span class="pre">xdev</span></code>.</p>
<img alt="_images/libqdma_data_structures.PNG" class="align-center" src="_images/libqdma_data_structures.PNG" />
<p><code class="docutils literal notranslate"><span class="pre">xdev</span></code> is associated with the driver level configuration called <code class="docutils literal notranslate"><span class="pre">qdma_dev_conf</span></code> and <code class="docutils literal notranslate"><span class="pre">descq</span></code> is associated with
queue level configuration called <code class="docutils literal notranslate"><span class="pre">qdma_queue_conf</span></code>.</p>
<p>The detailed containment of the data structures are captured below.</p>
<img alt="_images/qdma_main_datastructures.PNG" class="align-center" src="_images/qdma_main_datastructures.PNG" />
</div>
<div class="section" id="device-initialization">
<h2>Device Initialization<a class="headerlink" href="#device-initialization" title="Permalink to this headline">¶</a></h2>
<p>Each PF or VF is initialized when <code class="docutils literal notranslate"><span class="pre">probe</span></code> is invoked by Linux kernel upon identifying the corresponding PCI device.</p>
<ol class="arabic">
<li><p class="first">Identifying the Master PF</p>
<blockquote>
<div><p>QDMA driver enables the users of the QDMA IP to choose one of the available PFs as master_pf.
This is a software only feature and the HW does not have any knowledge on master_pf. HW treates all the PFs as equal.</p>
<p>This PF has the following special software controlled previllages:</p>
<ul class="simple">
<li>Programming the global CSR(Control and Status Registers)</li>
<li>Handling the errors reported by HW using a dedicated interrupt vector</li>
</ul>
<p>While inserting the QDMA Linux driver, user can pass the choice of the PF number as master_pf for a device
using device BDF(BBDDF- PCI Bus, Device, Function) number.</p>
</div></blockquote>
</li>
<li><p class="first">Enable the DMA capability in the device</p>
<blockquote>
<div><p>Marks all PCI regions associated with PCI device (pdev) as being reserved and enables the DMA capability of the device.</p>
</div></blockquote>
</li>
<li><p class="first">Mapping the PCI Bars</p>
<blockquote>
<div><p>QDMA IP design provides the QDMA functionality in AXI DMA Bar. It needs to have a User Bar(AXI4Lite Bar) to realize
the QDMA functionality and can have a optional Bypass Bar(AXI Bridge Master Bar) to achieve the bypass functionality.
These are 64bit bars.</p>
<p>For configuring the queues available in the system, QDMA driver needs to identify the DMA Bar from the avilable
PCI Bars of the device and memory map them to the Host system.</p>
<p>The config bar needs to be passed to the driver using module_params:</p>
<ul class="simple">
<li>Multiple devices can be connected to a single Host system</li>
<li>Each device has 4 PF’s hence module parameter shall support to pass 4 different values for each PF</li>
<li>All the VF’s corresponding to a PF has the same config bar hence module parameter shall support to pass</li>
</ul>
<p>4 different values for each PF’s VF group</p>
<p>Based on the above points, <code class="docutils literal notranslate"><span class="pre">config_bar</span></code> module parameter is composed as array of values where each entry supports to
pass config bar information for one device’s PF or PF’s VF.</p>
<p>config_bar=&lt;bus_num&gt;&lt;config_bar_PF0&gt;&lt;config_bar_PF1&gt;&lt;config_bar_PF2&gt;&lt;config_bar_PF3&gt;
Ex: config_bar_pf=0x0902020202,0x0a00000000</p>
<p>QDMA IP user has the flexibility to configure any of the PCI Bars as DMA Bar from Vivado GUI when creating QDMA IP design.
To indicate the DMA Bar QDMA IP programs 0x0 register with 0x1FD3 identifier.
Driver can find DMA BAR by reading address 0x0 from each BAR and check for (return value &amp; 0xFFFF0000) &gt;&gt; 16 == 0x1fd3 to find DMA BAR
By default BAR#0 is considered as DMA Bar by the Driver.
If user does not pass the <code class="docutils literal notranslate"><span class="pre">config_bar</span></code> module parameter, Driver tries to accessess the PCI Bar#0 and reads the 0x0 register.
If it does not have the 1FD3 idenfier, driver will fail to initialize the corresponding PF/VF.
If user passes the <code class="docutils literal notranslate"><span class="pre">config_bar</span></code> parameter, driver validates the given bar number with HW configured bar number and
if they match, the bar will be memory mapped.</p>
<p>Upon idenfying the DMA Bar, Driver identifies the User Bar using register
QDMA_GLBL2_PF_VF_BARLITE_INT((0x10C for PF and 0x1018 for VF) and the remaining bar is marked as bypass bar.</p>
</div></blockquote>
</li>
<li><p class="first">Create xdev and add to xdev list</p>
<blockquote>
<div><p>xdev is a libqdma level bok keeping structure which holds the information required for each function.
It is created from pdev and device configration updated according.
Once the xdev is created, it is added to the xdev list to keep track of the multiple functions.</p>
</div></blockquote>
</li>
<li><p class="first">Mark the device as online</p>
<blockquote>
<div><p>In this phase, The allocation of I/O ports and irqs is done via standard kernel functions.
First, it clears the HW context memory for all the queues if the current function is marked as a master_pf.
If the driver is loaded in interrupt mode, interrupt allocation for the set of available vectors for the function is fulfilled.
<code class="docutils literal notranslate"><span class="pre">qdev</span></code> is created along with all the descriptor rings for all the queues corresponding to the current function and
initialized with default values.
Global CSR registers are set with default values and mm channels are enabled.</p>
<p>If the current function is a VF, message box(mbox) is created and started to communicate with the parent PF.</p>
</div></blockquote>
</li>
</ol>
<img alt="_images/probe_one.PNG" class="align-center" src="_images/probe_one.PNG" />
</div>
<div class="section" id="device-de-initialization">
<h2>Device De-Initialization<a class="headerlink" href="#device-de-initialization" title="Permalink to this headline">¶</a></h2>
<p>During the driver de-inilization i.e when user performs “rmmmod”, all the devices managed by the driver are de-initialized by
invoking the “remove” handler.</p>
<p>During the device de-initialization, following operations are perfomed</p>
<ul class="simple">
<li>Configuration for all the queues corresponding to the device are cleared</li>
<li>The character device interface created for the device is released</li>
<li>PCI bars are unmapped</li>
<li>Set the device to offline and removed the device from the device list</li>
</ul>
<img alt="_images/remove_one.PNG" class="align-center" src="_images/remove_one.PNG" />
</div>
<div class="section" id="configure-the-queues-for-pf-vf">
<h2>Configure the Queues for PF/VF<a class="headerlink" href="#configure-the-queues-for-pf-vf" title="Permalink to this headline">¶</a></h2>
<p>During the driver initialization, the available queues is the system are evenly distributed across the available PFs.
Ex: if the device has 4 PFs and 2K queues, each PF is assigned with 512 queues.</p>
<p>QDMA driver provides a sysfs interface to change the default distribution of queues according to user choice.
<code class="docutils literal notranslate"><span class="pre">/sys/bus/pci/devices/&lt;pci_device_bdf&gt;/qdma/qmax</span></code> sysfs entry is created for each function to facilitate this user configuration.
User can set the required number of queues using this sysfs entry.</p>
<p>If the queues in the system needs to be allocated for VFs, master_pf has a sysfs entry <code class="docutils literal notranslate"><span class="pre">/sys/bus/pci/devices/&lt;pci_device_bdf&gt;/qdma/qmax_vfs</span></code>.
The queues configured for <code class="docutils literal notranslate"><span class="pre">qmax_vfs</span></code> is a pool of queues which will be used to distribute across all available VFs created for this device.
The remining queues are evenly distibuted across the PFs.</p>
<img alt="_images/set_qmax.PNG" class="align-center" src="_images/set_qmax.PNG" />
<img alt="_images/set_qmax_vfs.PNG" class="align-center" src="_images/set_qmax_vfs.PNG" />
</div>
<div class="section" id="add-a-queue">
<h2>Add a Queue<a class="headerlink" href="#add-a-queue" title="Permalink to this headline">¶</a></h2>
<p>QDMA Linux Driver exposes the <code class="docutils literal notranslate"><span class="pre">qdma_queue_add</span></code> API to add a queue to a function.
<code class="docutils literal notranslate"><span class="pre">dmactl</span></code> application provided along with QDMA driver enables
the user to add a queue. QDMA driver creates the queue handle for application and
a character dev interface to read and write to the queue.
FMAP programming is done for a function when a first queue is added to a function.
Once the FMAP programming is done, Global CSR(Configuration and Status Registers)
updating is freezed by the driver.</p>
<img alt="_images/xpdev_queue_add.PNG" class="align-center" src="_images/xpdev_queue_add.PNG" />
</div>
<div class="section" id="start-a-queue">
<h2>Start a Queue<a class="headerlink" href="#start-a-queue" title="Permalink to this headline">¶</a></h2>
<p>QDMA Linux Driver exposes the <code class="docutils literal notranslate"><span class="pre">qdma_queue_start</span></code> API to start an already added queue.
User can configure the queue using the various configuration options
provided through <code class="docutils literal notranslate"><span class="pre">dmactl</span></code> application. During queue start, driver creates the H2C/C2H descriptor rings,
configures the context for the queue, configures the interrupt context
if driver is loaded in interrupt mode, attaches a thread to the queue.</p>
<img alt="_images/xpdev_nl_queue_start.PNG" class="align-center" src="_images/xpdev_nl_queue_start.PNG" />
</div>
<div class="section" id="stop-a-queue">
<h2>Stop a Queue<a class="headerlink" href="#stop-a-queue" title="Permalink to this headline">¶</a></h2>
<p>QDMA Linux Driver exposes the <code class="docutils literal notranslate"><span class="pre">qdma_queue_stop</span></code> API to stop an already started queue.
All the queue resources are released and context is cleared.</p>
<img alt="_images/qdma_queue_stop.PNG" class="align-center" src="_images/qdma_queue_stop.PNG" />
</div>
<div class="section" id="delete-a-queue">
<h2>Delete a Queue<a class="headerlink" href="#delete-a-queue" title="Permalink to this headline">¶</a></h2>
<p>QDMA Linux Driver exposes the <code class="docutils literal notranslate"><span class="pre">qdma_queue_delete</span></code> API to delete an already added queue.
Queue handle is released and character device interface is deleted</p>
<img alt="_images/xpdev_queue_delete.PNG" class="align-center" src="_images/xpdev_queue_delete.PNG" />
</div>
<div class="section" id="read-write-from-a-queue">
<h2>Read/Write from a Queue<a class="headerlink" href="#read-write-from-a-queue" title="Permalink to this headline">¶</a></h2>
<p><code class="docutils literal notranslate"><span class="pre">dma_from_device</span></code> and <code class="docutils literal notranslate"><span class="pre">dma_to_device</span></code> utilities are provided with QDMA Linux driver to perform
a read(C2H) and write(H2C) operations on a queue.</p>
<img alt="_images/read_write_request.PNG" class="align-center" src="_images/read_write_request.PNG" />
</div>
</div>


           </div>
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="qdma_usecases.html" class="btn btn-neutral float-right" title="QDMA Linux Driver UseCases" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="libqdma_apis.html" class="btn btn-neutral" title="Libqdma APIs" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2018-2019, Xilinx, Inc.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'./',
            VERSION:'2018.3',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true
        };
    </script>
      <script type="text/javascript" src="_static/jquery.js"></script>
      <script type="text/javascript" src="_static/underscore.js"></script>
      <script type="text/javascript" src="_static/doctools.js"></script>

  

  
  
    <script type="text/javascript" src="_static/js/theme.js"></script>
  

  
  
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.StickyNav.enable();
      });
  </script>
   

</body>
</html>